{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, None, None, 3) 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)            (None, None, None, 64 1792        input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)            (None, None, None, 64 36928       block1_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block1_pool (MaxPooling2D)       (None, None, None, 64 0           block1_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv1 (Conv2D)            (None, None, None, 12 73856       block1_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv2 (Conv2D)            (None, None, None, 12 147584      block2_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)       (None, None, None, 12 0           block2_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv1 (Conv2D)            (None, None, None, 25 295168      block2_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv2 (Conv2D)            (None, None, None, 25 590080      block3_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv3 (Conv2D)            (None, None, None, 25 590080      block3_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)       (None, None, None, 25 0           block3_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv1 (Conv2D)            (None, None, None, 51 1180160     block3_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv2 (Conv2D)            (None, None, None, 51 2359808     block4_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv3 (Conv2D)            (None, None, None, 51 2359808     block4_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)       (None, None, None, 51 0           block4_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv1 (Conv2D)            (None, None, None, 51 2359808     block4_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv2 (Conv2D)            (None, None, None, 51 2359808     block5_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv3 (Conv2D)            (None, None, None, 51 2359808     block5_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "rpn_conv1 (Conv2D)               (None, None, None, 51 2359808     block5_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)                (None, None, 512)     0           rpn_conv1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "blstm (Bidirectional)            (None, None, 256)     492288      lambda_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)                (None, None, None, 25 0           blstm[0][0]                      \n",
      "                                                                   rpn_conv1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "lstm_fc (Conv2D)                 (None, None, None, 51 131584      lambda_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "rpn_class (Conv2D)               (None, None, None, 20 10260       lstm_fc[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "rpn_regress (Conv2D)             (None, None, None, 20 10260       lstm_fc[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "rpn_class_reshape (Lambda)       (None, None, 2)       0           rpn_class[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "rpn_regress_reshape (Lambda)     (None, None, 2)       0           rpn_regress[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 17,718,888\n",
      "Trainable params: 17,718,888\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.models import Model,Sequential\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.core import Reshape,Dense,Flatten,Permute,Lambda,Activation\n",
    "from keras.layers.wrappers import Bidirectional,TimeDistributed\n",
    "from keras.layers import Input\n",
    "from keras.optimizers import Adam,SGD\n",
    "from keras import backend as K\n",
    "from keras import regularizers \n",
    "import tensorflow as tf \n",
    "from keras.callbacks import EarlyStopping,ModelCheckpoint,Callback\n",
    "\n",
    "def rpn_loss_regr(y_true,y_pred):\n",
    "    \"\"\"\n",
    "    smooth L1 loss\n",
    "  \n",
    "    y_ture [1][HXWX9][3] (class,regr)\n",
    "    y_pred [1][HXWX9][2] (reger)\n",
    "    \"\"\"   \n",
    "    \n",
    "    sigma=9.0\n",
    "    \n",
    "    cls = y_true[0,:,0]\n",
    "    regr = y_true[0,:,1:3]\n",
    "    regr_keep = tf.where(K.equal(cls,1))[:,0]\n",
    "    regr_true = tf.gather(regr,regr_keep)\n",
    "    regr_pred = tf.gather(y_pred[0],regr_keep)\n",
    "    diff = tf.abs(regr_true-regr_pred)\n",
    "    less_one = tf.cast(tf.less(diff,1.0/sigma),'float32')\n",
    "    loss = less_one * 0.5 * diff**2 * sigma   + tf.abs(1-less_one) * (diff -0.5/sigma)\n",
    "    loss = K.sum(loss,axis=1)\n",
    "\n",
    "    return K.switch(tf.size(loss)>0,K.mean(loss),K.constant(0.0))\n",
    "\n",
    "def rpn_loss_cls(y_true,y_pred):\n",
    "    \"\"\"\n",
    "    softmax loss\n",
    "    \n",
    "    y_true [1][1][HXWX9] class\n",
    "    y_pred [1][HXWX9][2] class \n",
    "    \"\"\" \n",
    "    y_true = y_true[0][0]\n",
    "    cls_keep = tf.where(tf.not_equal(y_true,-1))[:,0]\n",
    "    cls_true = tf.gather(y_true,cls_keep)\n",
    "    cls_pred = tf.gather(y_pred[0],cls_keep)\n",
    "    cls_true = tf.cast(cls_true,'int64')\n",
    "    #loss = K.sparse_categorical_crossentropy(cls_true,cls_pred,from_logits=True)\n",
    "    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels = cls_true,logits=cls_pred)\n",
    "    return K.switch(tf.size(loss)>0,K.clip(K.mean(loss),0,10),K.constant(0.0))\n",
    "\n",
    "def nn_base(input,trainable):\n",
    "    base_model = VGG16(weights=None,include_top=False,input_shape = input)\n",
    "    base_model.load_weights('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')\n",
    "    if(trainable ==False):\n",
    "        for ly in base_model.layers:\n",
    "            ly.trainable = False\n",
    "    return base_model.input,base_model.get_layer('block5_conv3').output\n",
    "\n",
    "def reshape(x):\n",
    "    b = tf.shape(x)\n",
    "    x = tf.reshape(x,[b[0]*b[1],b[2],b[3]])\n",
    "    return x\n",
    "\n",
    "def reshape2(x):\n",
    "    x1,x2 = x\n",
    "    b = tf.shape(x2)\n",
    "    x = tf.reshape(x1,[b[0],b[1],b[2],256])\n",
    "    return x \n",
    "\n",
    "def reshape3(x):\n",
    "    b = tf.shape(x)\n",
    "    x = tf.reshape(x,[b[0],b[1]*b[2]*10,2])\n",
    "    return x \n",
    "\n",
    "def rpn(base_layers):\n",
    "    \n",
    "    x = Conv2D(512,(3,3),strides=(1,1),padding='same',activation='relu',\n",
    "               name='rpn_conv1')(base_layers)\n",
    "    \n",
    "    x1 = Lambda(reshape,output_shape=(None,512))(x) \n",
    "    \n",
    "    x2 = Bidirectional(GRU(128,return_sequences=True),name='blstm')(x1)\n",
    "\n",
    "    x3 = Lambda(reshape2,output_shape=(None,None,256))([x2,x])\n",
    "    x3 = Conv2D(512,(1,1),padding='same',activation='relu',name='lstm_fc')(x3)\n",
    "\n",
    "    cls = Conv2D(10*2,(1,1),padding='same',activation='linear',name='rpn_class')(x3)\n",
    "    regr = Conv2D(10*2,(1,1),padding='same',activation='linear',name='rpn_regress')(x3)\n",
    "    \n",
    "\n",
    "    cls = Lambda(reshape3,output_shape=(None,2),name='rpn_class_reshape')(cls)\n",
    "   \n",
    "    regr = Lambda(reshape3,output_shape=(None,2),name='rpn_regress_reshape')(regr)\n",
    "    \n",
    "    return cls,regr\n",
    "\n",
    "inp,nn = nn_base((None,None,3),trainable=True)\n",
    "cls,regr = rpn(nn)\n",
    "basemodel =  Model(inp,[cls,regr])\n",
    "basemodel.summary()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import utils\n",
    "xmlpath = 'E:\\BaiduNetdiskDownload\\VOCdevkit\\VOC2007\\Annotations'\n",
    "imgpath = 'E:\\BaiduNetdiskDownload\\VOCdevkit\\VOC2007\\JPEGImages'\n",
    "gen1 = utils.gen_sample(xmlpath,imgpath,1)\n",
    "gen2 = utils.gen_sample(xmlpath,imgpath,1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class losslog():\n",
    "    def __init__(self,path,txt):\n",
    "        with open(path,'a+') as f:\n",
    "            f.writelines(txt)\n",
    "\n",
    "class losshistroy(Callback):\n",
    "    def on_train_begin(self,logs={}):\n",
    "        self.losses=[]\n",
    "    def on_batch_end(self,batch,logs={}):\n",
    "        self.losses.append(logs.get('loss'))\n",
    "        txtloss= str(logs.get('rpn_class_reshape_loss'))+','+str(logs.get('rpn_regress_reshape_loss'))+'\\r\\n'\n",
    "        losslog('loss3.cvs',txtloss)\n",
    "        #print('rpn_class_reshape_loss:',logs.get('rpn_class_reshape_loss'),' ','rpn_regress_reshape_loss:',logs.get('rpn_regress_reshape_loss'))\n",
    "\n",
    "hisloss = losshistroy()\n",
    "\n",
    "checkpoint = ModelCheckpoint(r'E:\\deeplearn\\ctpn2018\\model\\weights-ctpnlstm-{epoch:02d}.hdf5',\n",
    "                           save_weights_only=True)\n",
    "earlystop = EarlyStopping(patience=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "utils.get_session(gpu_fraction=0.6)\n",
    "\n",
    "#sgd = SGD(0.0001,0.9,nesterov = True)\n",
    "adam = Adam(0.00001)\n",
    "basemodel.compile(optimizer = adam,\n",
    "                  loss = {'rpn_class_reshape':rpn_loss_cls,'rpn_regress_reshape':rpn_loss_regr},\n",
    "                  loss_weights={'rpn_class_reshape':1.0,'rpn_regress_reshape':1.0}\n",
    "                  )\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py:95: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "6000/6000 [==============================] - 2843s - loss: 0.4414 - rpn_class_reshape_loss: 0.2699 - rpn_regress_reshape_loss: 0.1714  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\Anaconda3\\lib\\site-packages\\keras\\callbacks.py:496: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: rpn_class_reshape_loss,rpn_regress_reshape_loss,loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 2/20\n",
      "6000/6000 [==============================] - 2851s - loss: 0.2657 - rpn_class_reshape_loss: 0.1327 - rpn_regress_reshape_loss: 0.1330  \n",
      "Epoch 3/20\n",
      "6000/6000 [==============================] - 2861s - loss: 0.2225 - rpn_class_reshape_loss: 0.1071 - rpn_regress_reshape_loss: 0.1154  \n",
      "Epoch 4/20\n",
      "6000/6000 [==============================] - 2852s - loss: 0.1984 - rpn_class_reshape_loss: 0.0926 - rpn_regress_reshape_loss: 0.1058  \n",
      "Epoch 5/20\n",
      "6000/6000 [==============================] - 2849s - loss: 0.1814 - rpn_class_reshape_loss: 0.0838 - rpn_regress_reshape_loss: 0.0976  \n",
      "Epoch 6/20\n",
      "6000/6000 [==============================] - 2856s - loss: 0.1672 - rpn_class_reshape_loss: 0.0762 - rpn_regress_reshape_loss: 0.0909  \n",
      "Epoch 7/20\n",
      "6000/6000 [==============================] - 2848s - loss: 0.1556 - rpn_class_reshape_loss: 0.0698 - rpn_regress_reshape_loss: 0.0858  \n",
      "Epoch 8/20\n",
      "6000/6000 [==============================] - 2861s - loss: 0.1445 - rpn_class_reshape_loss: 0.0637 - rpn_regress_reshape_loss: 0.0808  \n",
      "Epoch 9/20\n",
      "6000/6000 [==============================] - 2858s - loss: 0.1357 - rpn_class_reshape_loss: 0.0593 - rpn_regress_reshape_loss: 0.0764  \n",
      "Epoch 10/20\n",
      "6000/6000 [==============================] - 2962s - loss: 0.1280 - rpn_class_reshape_loss: 0.0553 - rpn_regress_reshape_loss: 0.0727  \n",
      "Epoch 11/20\n",
      "6000/6000 [==============================] - 2850s - loss: 0.1201 - rpn_class_reshape_loss: 0.0511 - rpn_regress_reshape_loss: 0.0690  \n",
      "Epoch 12/20\n",
      "6000/6000 [==============================] - 2860s - loss: 0.1134 - rpn_class_reshape_loss: 0.0479 - rpn_regress_reshape_loss: 0.0655  \n",
      "Epoch 13/20\n",
      "6000/6000 [==============================] - 2840s - loss: 0.1081 - rpn_class_reshape_loss: 0.0449 - rpn_regress_reshape_loss: 0.0632  \n",
      "Epoch 14/20\n",
      "6000/6000 [==============================] - 2834s - loss: 0.1022 - rpn_class_reshape_loss: 0.0423 - rpn_regress_reshape_loss: 0.0600  \n",
      "Epoch 15/20\n",
      "6000/6000 [==============================] - 2839s - loss: 0.0967 - rpn_class_reshape_loss: 0.0393 - rpn_regress_reshape_loss: 0.0574  \n",
      "Epoch 16/20\n",
      "6000/6000 [==============================] - 2869s - loss: 0.0925 - rpn_class_reshape_loss: 0.0373 - rpn_regress_reshape_loss: 0.0552  \n",
      "Epoch 17/20\n",
      "6000/6000 [==============================] - 2875s - loss: 0.0883 - rpn_class_reshape_loss: 0.0354 - rpn_regress_reshape_loss: 0.0529  \n",
      "Epoch 18/20\n",
      "6000/6000 [==============================] - 2871s - loss: 0.0848 - rpn_class_reshape_loss: 0.0339 - rpn_regress_reshape_loss: 0.0510  \n",
      "Epoch 19/20\n",
      "6000/6000 [==============================] - 2872s - loss: 0.0806 - rpn_class_reshape_loss: 0.0320 - rpn_regress_reshape_loss: 0.0486  \n",
      "Epoch 20/20\n",
      "6000/6000 [==============================] - 2872s - loss: 0.0783 - rpn_class_reshape_loss: 0.0309 - rpn_regress_reshape_loss: 0.0473  \n"
     ]
    }
   ],
   "source": [
    "res = basemodel.fit_generator(gen1,6000,epochs =20,verbose=1,callbacks=[checkpoint,earlystop,hisloss])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
